package fr.inserm.um915.jimpute.model;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

import com.sleepycat.bind.tuple.IntegerBinding;
import com.sleepycat.bind.tuple.StringBinding;
import com.sleepycat.je.Cursor;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.LockMode;
import com.sleepycat.je.OperationStatus;
import com.sleepycat.je.SecondaryConfig;
import com.sleepycat.je.SecondaryDatabase;
import com.sleepycat.je.SecondaryKeyCreator;
import com.sleepycat.je.Transaction;

public class Model
	{
	private static final Logger LOG=Logger.getLogger("fr.inserm.um915.jimpute");
	private static final String DB_NAME2MARKER="rs2marker";
	private static final String DB_POS2MARKER="pos2marker";
	private static final String DB_INDEX2INDIVIDUAL="index2individual";
	private static final String DBNAME2INDIVIDUAL="name2individual";
	private static final String DB_POS2COMBINED="pos2combined";
	private static final String DB_RS2RSMERGEARCH="rs2rsMergeArch";
	private static final String DB_RS2DBSNP_POS="rs2dbsnpPositions";
	private static final String DB_DBSNP2RSID="dbsnpPositions2rs";
	
	/** should we cleanup on close */
	private boolean cleanupOnClose=false;
	/** environment */
	private Environment environment;
	/** assure close on exit */
	private Thread shutdownHook;
	/** map rs## to Marker */
	private Database name2marker=null;
	/** map position to Marker */
	private SecondaryDatabase pos2marker=null;
	/** map index to individual */
	private Database index2individual=null;
	/** map name to individual */
	private SecondaryDatabase name2individual=null;
	/** map position to combined */
	private Database pos2combined=null;
	/** map rs## to MergeArch */
	private Database rs2rsMergeArch=null;
	/** map rs to dbsnp position (UCSC) */
	private Database rs2dbsnpPositions=null;
	/** map dbsnp pos to rsid */
	private Database posdbsnp2rsid=null;
	/** userDB */
	private Database extraTmpDatabase=null;
	
	
	static class Factory extends ModelFactory
		{
		@Override
		public Model open() throws IOException
			{
			if(Chromosome.values().length>=Byte.MAX_VALUE)
				{
				throw new IllegalStateException("too many chromosomes");
				}
			Model m= new Model(this);
			return m;
			}
		}
	
	private Model(Factory factory) throws IOException
		{
		Transaction txn=null;
		close();
		this.cleanupOnClose=!factory.isReadOnly();
		try
			{
			this.shutdownHook=new Thread()
				{
				@Override
				public void run()
					{
					Model.this.close();
					}
				};
			LOG.info("opening "+factory.getDbHome());
			EnvironmentConfig envCfg=new EnvironmentConfig();
			envCfg.setAllowCreate(!factory.isReadOnly());
			envCfg.setReadOnly(false);//for temporary
			//if(!factory.isReadOnly()) envCfg.setCachePercent(80);
			envCfg.setConfigParam(EnvironmentConfig.LOG_FILE_MAX,"500000000");
			/** see http://www.oracle.com/technetwork/database/berkeleydb/je-faq-096044.html#CanBerkeleyDBJavaEditionuseaNFSSANorotherremotesharednetworkfilesystemforanenvironment */
			//envCfg.setConfigParam(EnvironmentConfig.LOG_USE_ODSYNC, "true");
			
			if(factory.isIgnoreChecksumRead())
				{
				envCfg.setConfigParam(EnvironmentConfig.LOG_CHECKSUM_READ, "false");
				}
			
			this.environment=new Environment(factory.getDbHome(), envCfg);
			
			if(!factory.isReadOnly())
				{
				for(String db:this.environment.getDatabaseNames())
					{
					LOG.info("drop "+db);
					this.environment.truncateDatabase(txn, db, false);
					}
				
				}
			
			
			
			DatabaseConfig cfg=new DatabaseConfig();
			cfg.setAllowCreate(!factory.isReadOnly());
			cfg.setReadOnly(factory.isReadOnly());
			//cfg.setDeferredWrite(!readOnly);
			this.name2marker = this.environment.openDatabase(txn, DB_NAME2MARKER, cfg);
			
			
			
			SecondaryConfig cfg2=new SecondaryConfig();
			cfg2.setAllowCreate(!factory.isReadOnly());
			cfg2.setReadOnly(factory.isReadOnly());
			cfg2.setBtreeComparator(ChromPosition.ChromPositionComparator.class);
			cfg2.setSortedDuplicates(true);
			cfg2.setKeyCreator(new SecondaryKeyCreator()
				{
				@Override
				public boolean createSecondaryKey(
						SecondaryDatabase secondary,
	                    DatabaseEntry key,
	                    DatabaseEntry data,
	                    DatabaseEntry result)
					{
					Marker m=Marker.BINDING.entryToObject(data);
					ChromPosition.BINDING.objectToEntry(m.getPosition(), result);
					return true;
					}
				});
			//cfg2.setDeferredWrite(!readOnly);
			this.pos2marker=this.environment.openSecondaryDatabase(txn, DB_POS2MARKER, name2marker,cfg2);
			
			
			/* INDIVIDUALS */
			
			cfg=new DatabaseConfig();
			cfg.setAllowCreate(!factory.isReadOnly());
			cfg.setReadOnly(factory.isReadOnly());
			//cfg.setDeferredWrite(!readOnly);
			this.index2individual = this.environment.openDatabase(txn, DB_INDEX2INDIVIDUAL, cfg);
			
			
			
			cfg2=new SecondaryConfig();
			cfg2.setAllowCreate(!factory.isReadOnly());
			cfg2.setReadOnly(factory.isReadOnly());
			cfg2.setSortedDuplicates(false);//FALSE
			cfg2.setKeyCreator(new SecondaryKeyCreator()
				{
				@Override
				public boolean createSecondaryKey(
						SecondaryDatabase secondary,
	                    DatabaseEntry key,
	                    DatabaseEntry data,
	                    DatabaseEntry result)
					{
					Individual m=Individual.BINDING.entryToObject(data);
					StringBinding.stringToEntry(m.getName(), result);
					return true;
					}
				});
			this.name2individual=this.environment.openSecondaryDatabase(txn, DBNAME2INDIVIDUAL, index2individual,cfg2);
			
			/* Combined */
			cfg=new DatabaseConfig();
			cfg.setAllowCreate(!factory.isReadOnly());
			cfg.setReadOnly(factory.isReadOnly());
			cfg.setBtreeComparator(ChromPosition.ChromPositionComparator.class);
			this.pos2combined = this.environment.openDatabase(txn, DB_POS2COMBINED, cfg);
	
			/* RsmergArch */
			cfg=new DatabaseConfig();
			cfg.setAllowCreate(!factory.isReadOnly());
			cfg.setReadOnly(factory.isReadOnly());
			this.rs2rsMergeArch = this.environment.openDatabase(txn, DB_RS2RSMERGEARCH, cfg);
	
			
			/* dbsnp */
			cfg=new DatabaseConfig();
			cfg.setAllowCreate(!factory.isReadOnly());
			cfg.setReadOnly(factory.isReadOnly());
			this.rs2dbsnpPositions = this.environment.openDatabase(txn, DB_RS2DBSNP_POS, cfg);
			
			cfg=new DatabaseConfig();
			cfg.setAllowCreate(!factory.isReadOnly());
			cfg.setReadOnly(factory.isReadOnly());
			cfg.setSortedDuplicates(true);
			cfg.setBtreeComparator(BuildChromPosition.BuildChromPositionComparator.class);
			this.posdbsnp2rsid=this.environment.openDatabase(txn, DB_DBSNP2RSID,cfg);
			
			
			if(this.shutdownHook==null) throw new NullPointerException();
			Runtime.getRuntime().addShutdownHook(this.shutdownHook);
			}
		catch(Exception err)
			{
			close();
			throw new IOException(err);
			}
		
		}
	
	public Database getExtraDatabase(Transaction txn)
		{
		if(this.extraTmpDatabase==null)
			{
			DatabaseConfig cfg=new DatabaseConfig();
			cfg.setAllowCreate(true);
			cfg.setReadOnly(false);
			cfg.setTemporary(true);
			this.extraTmpDatabase=this.environment.openDatabase(txn, "_tmp"+System.currentTimeMillis()+"_"+Math.random(), cfg);
			}
		return this.extraTmpDatabase;
		}
	
	public Marker findMarkerByName(Transaction txn,int rsId)
		{
		return findMarkerByName(txn,"rs"+rsId);
		}
	
	public Marker findMarkerByName(Transaction txn,String name)
		{
		DatabaseEntry key=new DatabaseEntry();
		DatabaseEntry data=new DatabaseEntry();
		StringBinding.stringToEntry(name, key);
		if(this.name2marker.get(txn, key, data, LockMode.DEFAULT)!=OperationStatus.SUCCESS)
			{
			return null;
			}
		return Marker.BINDING.entryToObject(data);
		}
	
	public RsMergeArch findRsMergeByName(Transaction txn,String rsId)
		{
		if(rsId==null || !rsId.startsWith("rs")) return null;
		try
			{
			return findRsMergeByName(txn,Integer.parseInt(rsId.substring(2)));
			}
		catch(NumberFormatException err)
			{
			return null;
			}
		}
	
	public RsMergeArch findRsMergeByName(Transaction txn,int rsId)
		{
		DatabaseEntry key=new DatabaseEntry();
		DatabaseEntry data=new DatabaseEntry();
		IntegerBinding.intToEntry(rsId, key);
		if(this.rs2rsMergeArch.get(txn, key, data, LockMode.DEFAULT)!=OperationStatus.SUCCESS)
			{
			return null;
			}
		return RsMergeArch.BINDING.entryToObject(data);
		}
	
	public List<BuildChromPosition> findRsPositionsByName(Transaction txn,Build build,String rsId)
		{
		if(!rsId.startsWith("rs")) throw new IllegalArgumentException(rsId);
		return findRsPositionsByName(txn,build,Integer.parseInt(rsId.substring(2)));
		}

	
	public List<BuildChromPosition> findRsPositionsByName(Transaction txn,Build build,int rsId)
		{
		List<BuildChromPosition> L0=findRsPositionsByName(txn, rsId);
		if(L0==null) return L0;
		ArrayList<BuildChromPosition> L=new ArrayList<BuildChromPosition> (L0);
		int i=0;
		while(i<L.size())
			{
			if(L.get(i).getBuildIdx()==build.ordinal())
				{
				++i;
				continue;
				}
			L.remove(i);
			}
		return L;
		}
	
	
	public List<BuildChromPosition> findRsPositionsByName(Transaction txn,String rsId)
		{
		if(!rsId.startsWith("rs")) throw new IllegalArgumentException(rsId);
		return findRsPositionsByName(txn,Integer.parseInt(rsId.substring(2)));
		}

	
	public List<BuildChromPosition> findRsPositionsByName(Transaction txn,int rsId)
		{
		DatabaseEntry key=new DatabaseEntry();
		DatabaseEntry data=new DatabaseEntry();
		IntegerBinding.intToEntry(rsId, key);
		if(this.rs2dbsnpPositions.get(txn, key, data, LockMode.DEFAULT)!=OperationStatus.SUCCESS)
			{
			return null;
			}
		return BuildChromPosition.ARRAY_BINDING.entryToObject(data);
		}

	
	public Set<Integer> findRsIdByPosition(Transaction txn,BuildChromPosition pos)
		{
		Set<Integer> rsids=new HashSet<Integer>();
		DatabaseEntry key=new DatabaseEntry();
		DatabaseEntry data=new DatabaseEntry();
		Cursor c=null;
		BuildChromPosition.BINDING.objectToEntry(pos, key);
		try {
			c=this.posdbsnp2rsid.openCursor(txn, null);
			boolean first=true;
			for(;;)
				{
				OperationStatus status;
				if(first)
					{
					status=c.getSearchKey(key, data, LockMode.DEFAULT);
					first=false;
					}
				else
					{
					status=c.getNextDup(key, data, LockMode.DEFAULT);
					}
				if(status!=OperationStatus.SUCCESS) break;
				rsids.add(IntegerBinding.entryToInt(data));
				}
			return rsids;
			} 
		catch (Exception e)
			{
			throw new RuntimeException(e);
			}
		finally
			{
			if(c!=null) c.close();
			}
		}
	
	@Override
	protected void finalize() throws Throwable {
		close();
		super.finalize();
		}
	
	public void close()
		{
		
		if(extraTmpDatabase!=null)
			{
			try { this.extraTmpDatabase.close();} catch(Throwable err) {err.printStackTrace();}
			this.extraTmpDatabase=null;
			}
		
		if(pos2marker!=null)
			{
			try { this.pos2marker.close();} catch(Throwable err) {err.printStackTrace();}
			this.pos2marker=null;
			}

		if(name2marker!=null)
			{
			try { this.name2marker.close();} catch(Throwable err) {err.printStackTrace();}
			this.name2marker=null;
			}
		
		if(name2individual!=null)
			{
			try { this.name2individual.close();} catch(Throwable err) {err.printStackTrace();}
			this.name2individual=null;
			}
	
		if(index2individual!=null)
			{
			try { this.index2individual.close();} catch(Throwable err) {err.printStackTrace();}
			this.index2individual=null;
			}
		
		if(pos2combined!=null)
			{
			try { this.pos2combined.close();} catch(Throwable err) {err.printStackTrace();}
			this.pos2combined=null;
			}
		
		if(rs2rsMergeArch!=null)
			{
			try { this.rs2rsMergeArch.close();} catch(Throwable err) {err.printStackTrace();}
			this.rs2rsMergeArch=null;
			}
		
		if(posdbsnp2rsid!=null)
			{
			try { this.posdbsnp2rsid.close();} catch(Throwable err) {err.printStackTrace();}
			this.posdbsnp2rsid=null;
			}
		
		if(rs2dbsnpPositions!=null)
			{
			try { this.rs2dbsnpPositions.close();} catch(Throwable err) {err.printStackTrace();}
			this.rs2dbsnpPositions=null;
			}
		
		if(environment!=null)
			{
			LOG.info("closing");
			if( this.cleanupOnClose && 
				!this.environment.getConfig().getReadOnly())
				{
				try {LOG.info("sync");this.environment.sync();} catch(Throwable err) {err.printStackTrace();}
				try {LOG.info("cleanLog");this.environment.cleanLog();} catch(Throwable err) {err.printStackTrace();}
				try {LOG.info("compress");this.environment.compress();} catch(Throwable err) {err.printStackTrace();}
				}
			try { this.environment.close();} catch(Throwable err) {err.printStackTrace();}
			this.environment=null;
			}
		if(this.shutdownHook!=null)
			{
			LOG.info("remove hook");
			Runtime.getRuntime().removeShutdownHook(this.shutdownHook);
			this.shutdownHook=null;
			}
		}
	public void loadImputeFolder(Transaction txn,File baseDir)
		throws IOException
		{
		LOG.info("Reading "+baseDir);
		File files[]=baseDir.listFiles(new FileFilter()
			{
			@Override
			public boolean accept(File f)
				{
				return f.isFile() && f.getName().endsWith(".sample");
				}
			});
		if(files.length!=1) throw new IOException("Expected one sample file but got "+Arrays.asList(files));
		loadSamples(txn,files[0]);
		
		files=baseDir.listFiles(new FileFilter()
			{
			@Override
			public boolean accept(File f)
				{
				return f.isFile() && f.getName().endsWith(".hap.gz");
				}
			});
		for(File hap: files)
			{
			File legendFile =new File(hap.getParentFile(),
					hap.getName().substring(0,hap.getName().length()-7)+".legend.gz"
					);
			if(!legendFile.exists())
				{
				throw new IOException("Cannot find legend file "+legendFile+ " for "+hap);
				}
			Chromosome chrom=null;
			
			for(Chromosome c: Chromosome.values())
				{
				String chromName=c.name();
				if(c==Chromosome.chrX) chromName="chrX_nonPAR";
				int i=hap.getName().indexOf(chromName+"_");
				if(i==-1) continue;
				if(chrom!=null) throw new IllegalStateException(hap.getName()+" "+chrom+" "+c);
				chrom=c;
				}
			if(chrom==null)
				{
				throw new IllegalStateException("Cannot get chrom for "+hap);
				}
			loadHapAndLegend(txn,(byte)chrom.ordinal(),hap,legendFile);
			}
		
		files=baseDir.listFiles(new FileFilter()
			{
			@Override
			public boolean accept(File f)
				{
				return f.isFile() &&
						(f.getName().endsWith(".txt") || f.getName().endsWith(".txt.gz"))
						&& f.getName().contains("_combined_");
				}
			});
		for(File file: files)
			{
		
			Chromosome chrom=null;
			
			for(Chromosome c: Chromosome.values())
				{
				String chromName=c.name();
				if(c==Chromosome.chrX) chromName="chrX_nonPAR";
				int i=file.getName().indexOf(chromName+"_");
				if(i==-1) continue;
				if(chrom!=null) throw new IllegalStateException(file.getName()+" "+c+" "+chrom);
				chrom=c;
				}
			if(chrom==null)
				{
				throw new IllegalStateException("Cannot get chrom for "+file);
				}
			loadCombined(txn,(byte)chrom.ordinal(),file);
			}
		
		}
	
	
	public Combined getCombinedByPosition(
			Transaction txn,
			ChromPosition pos
			) 
		{
		DatabaseEntry key=new DatabaseEntry();
		DatabaseEntry data=new DatabaseEntry();
		ChromPosition.BINDING.objectToEntry(pos, key);
		if(this.pos2combined.get(txn, key, data,LockMode.DEFAULT)!=OperationStatus.SUCCESS) return null;
		return Combined.BINDING.entryToObject(data);
		}
	
	private void loadCombined(Transaction txn,
			byte chromosome,
			File file
			)throws IOException
		{
		LOG.info("read "+file);
		int nRead=0;
		Pattern delim=Pattern.compile("[ ]");
		BufferedReader in=IOUtils.openReader(file);
		String line;
		while((line=in.readLine())!=null)
			{
			if(line.isEmpty() || line.startsWith("position")) continue;
			String tokens[]=delim.split(line);
			if(tokens.length<3) throw new IllegalStateException("Boum "+file+" "+line);
			
			int pos=Integer.parseInt(tokens[0]);
			ChromPosition cp=new ChromPosition(chromosome,pos);
			
			Combined comb=new Combined(
					Double.parseDouble(tokens[1]),
					Double.parseDouble(tokens[2])
					);
			DatabaseEntry key=new DatabaseEntry();
			DatabaseEntry data=new DatabaseEntry();
			
			ChromPosition.BINDING.objectToEntry(cp,key);
			Combined.BINDING.objectToEntry(comb, data);
			
			if(this.pos2combined.put(txn, key, data)!=OperationStatus.SUCCESS)
				{
				throw new IOException("Cannot insert "+line);
				}
			if((nRead++)%10000==0)
				{
				LOG.info(""+nRead+" "+line);
				}
			}
		
		in.close();
		}
	
	private void loadHapAndLegend(
			Transaction txn,
			byte chromosome,
			File hapFile,
			File legendFile
			)
			throws IOException
			{
			LOG.info("read "+hapFile+" and "+legendFile);
			int nRead=0;
			Pattern delim=Pattern.compile("[ ]");
			BufferedReader hapIn=new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(hapFile))));
			BufferedReader legIn=new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(legendFile))));
			String line=legIn.readLine();
			if(line==null) throw new IOException("Cannot read header for legend file "+legendFile);
			int population_start_column=-1;
			if(line.startsWith("id position a0 a1 src"))
				{
				population_start_column=5;
				}
			else if(line.startsWith("id position a0 a1 type source"))
				{
				population_start_column=6;
				}
			else
				{
				throw new IOException("not starting with id... "+line+" "+legendFile);
				}
			String header[]=delim.split(line);
			List<Marker.Population> populations=new ArrayList<Marker.Population>(header.length-population_start_column);
			for(int i=population_start_column;i<header.length;++i)
				{
				String h=header[i].replace('.', '_');
				Marker.Population pop=Marker.Population.valueOf(h);
				populations.add(pop);
				}
			while((line=legIn.readLine())!=null)
				{
				String line2=hapIn.readLine();
				if(line2==null) throw new IllegalStateException("Boum "+hapFile+" "+legendFile);
				String tokens[]=delim.split(line);
				if(tokens.length!=header.length) throw new IllegalStateException(line);
				if(findMarkerByName(txn, tokens[0])!=null)
					{
					throw new IllegalStateException("Duplicate marker name:"+line);
					}
				
				Marker m=new Marker();
				m.setName(tokens[0]);
				m.setPosition(new ChromPosition(chromosome, Integer.parseInt(tokens[1])));
				m.setA0(tokens[2]);
				m.setA1(tokens[3]);
				for(int i=4;i< tokens.length;++i)
					{
					m.getPopulationFrequencies().put(populations.get(i-4), Float.parseFloat(tokens[i]));
					}
				byte bitSet[]=new byte[(line2.length()+1)/2];
				int i=0;
				int bitIndex=0;
				for(;;)
					{
					char c1=line2.charAt(i);
					if(!(c1=='0' || c1=='1'|| c1=='-')) throw new IllegalStateException(line2);
					i++;
					if(line2.charAt(i)!=' ') throw new IllegalStateException(line2);
					i++;
					char c2=line2.charAt(i);
					if(!(c2=='0' || c2=='1'|| c2=='-')) throw new IllegalStateException(line2);
					i++;
					if(i==line2.length()) break;
					if(line2.charAt(i)!=' ') throw new IllegalStateException(line2);
					i++;
					bitSet[bitIndex++]=(byte)c1;
					bitSet[bitIndex++]=(byte)c1;;
					}
				m.setGenotypes(bitSet);
				
				DatabaseEntry key=new DatabaseEntry();
				DatabaseEntry data=new DatabaseEntry();
				
				StringBinding.stringToEntry(m.getName(),key);
				Marker.BINDING.objectToEntry(m, data);
				
				if(this.name2marker.put(txn, key, data)!=OperationStatus.SUCCESS)
					{
					throw new IOException("Cannot insert "+line);
					}
				if((nRead++)%10000==0)
					{
					LOG.info(m.getName()+" "+nRead);
					}	
				}
			String line2=hapIn.readLine();
			if(line2!=null && !line2.isEmpty()) throw new IllegalStateException("Boum "+hapFile+" "+legendFile+" line2");
			
			
			hapIn.close();
			legIn.close();
			}
	
	private void loadSamples(Transaction txn,File f)
		throws IOException
		{
		LOG.info("Read "+f);
		Pattern delim=Pattern.compile("[ ]");
		BufferedReader in=new BufferedReader(new FileReader(f));
		String line;
		int index=0;
		while((line=in.readLine())!=null)
			{
			if(line.isEmpty() || line.startsWith("sample")) continue;
			String tokens[]=delim.split(line);
			if(tokens.length<4) throw new IOException("bad line "+line+" in "+f);
			String name=tokens[0];
			Individual.Population pop=Individual.Population.valueOf(tokens[1]);
			Individual.Group g=Individual.Group.valueOf(tokens[2]);
			int sex=Integer.parseInt(tokens[3]);
			if(!(sex==1 || sex==2)) throw new IOException("bad sex in "+line);
			Individual indi=new Individual(index, name, pop, g, (byte)sex);
			
			DatabaseEntry key=new DatabaseEntry();
			DatabaseEntry data=new DatabaseEntry();
			IntegerBinding.intToEntry(index, key);
			Individual.BINDING.objectToEntry(indi,data);
			if(this.index2individual.put(txn, key, data)!=OperationStatus.SUCCESS)
				{
				throw new IOException("Cannot insert "+line);
				}
			index++;
			}
		in.close();
		}
	
	public void loadRsMerge(Transaction txn,File f) throws IOException
		{
		LOG.info("Read "+f);
		int nRead=0;
		Pattern delim=Pattern.compile("[\t]");
		BufferedReader in=new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f))));
		String line;
		while((line=in.readLine())!=null)
			{
			if(line.isEmpty()) continue;
			String tokens[]=delim.split(line);
			if(tokens.length<7) throw new IOException("bad line "+line+" in "+f+" "+tokens.length);
			RsMergeArch o=new RsMergeArch();
			o.setRsHigh(Integer.parseInt(tokens[0]));
			o.setRsLow(Integer.parseInt(tokens[1]));
			o.setBuildId(Short.parseShort(tokens[2]));
			if(tokens[3].isEmpty())
				{
				o.setOrient((byte)-1);
				}
			else if(tokens[3].equals("0"))
				{
				o.setOrient((byte)0);
				}
			else if(tokens[3].equals("1"))
				{
				o.setOrient((byte)1);
				}
			else
				{
				throw new IllegalArgumentException(" "+f+" "+line);
				}
			o.setRsCurrent(Integer.parseInt(tokens[6]));
			if(tokens.length<8 || tokens[7].isEmpty())
				{
				o.setOrien2Current((byte)-1);
				}
			else if(tokens[7].equals("0"))
				{
				o.setOrien2Current((byte)0);
				}
			else if(tokens[7].equals("1"))
				{
				o.setOrien2Current((byte)1);
				}
			else
				{
				throw new IllegalArgumentException(" "+f+" "+line);
				}
			
			
			DatabaseEntry key=new DatabaseEntry();
			DatabaseEntry data=new DatabaseEntry();
			IntegerBinding.intToEntry(o.getRsHigh(), key);
			RsMergeArch.BINDING.objectToEntry(o,data);
			if(this.rs2rsMergeArch.put(txn, key, data)!=OperationStatus.SUCCESS)
				{
				throw new IOException("Cannot insert "+line);
				}
			if((nRead++)%100000==0)
				{
				LOG.info("row ("+nRead+") : "+line);
				}
			}
		in.close();	
		}
	void loadDbSnp(Transaction txn,Build build,String url) throws IOException
		{
		LOG.info(url);
		int count=0;
		Pattern tab=Pattern.compile("[\t]");
		BufferedReader in=IOUtils.openUrl(url);
		String line;
		while((line=in.readLine())!=null)
			{
			if(++count%100000==0)
				{
				LOG.info(line);
				}
			String tokens[]=tab.split(line,6);
			Chromosome c=Chromosome.valueOf(tokens[1]);
			String rs=tokens[4];
			if(!rs.startsWith("rs")) throw new IOException(line);
			int rsId=Integer.parseInt(rs.substring(2));
			int chromStart=Integer.parseInt(tokens[2]);
			DatabaseEntry key=new DatabaseEntry();
			DatabaseEntry data=new DatabaseEntry();
			IntegerBinding.intToEntry(rsId, key);
			List<BuildChromPosition> positions=null;
			
			
			if(this.rs2dbsnpPositions.get(txn, key, data,LockMode.DEFAULT)==OperationStatus.SUCCESS)
				{
				positions=BuildChromPosition.ARRAY_BINDING.entryToObject(data);
				}
			else
				{
				positions=new ArrayList<BuildChromPosition>(1);
				}
			BuildChromPosition newpos=new BuildChromPosition((byte)build.ordinal(),(byte) c.ordinal(), chromStart);
			positions.add(newpos);
			
			
			BuildChromPosition.ARRAY_BINDING.objectToEntry(positions,data);
			if(this.rs2dbsnpPositions.put(txn, key, data)!=OperationStatus.SUCCESS)
				{
				throw new IOException("Cannot insert "+line);
				}
			
			data=new DatabaseEntry();
			BuildChromPosition.BINDING.objectToEntry(newpos, data);
			if(this.posdbsnp2rsid.put(txn, data,key)!=OperationStatus.SUCCESS)
				{
				throw new IOException("Cannot insert2 "+line);
				}
			}
		in.close();
		}
	
	}
